﻿using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Net;
using System.Collections;
using System.Text.RegularExpressions;
using System.Threading;

namespace Crawler
{
    class Crawlerpro
    {
        public Hashtable urls = new Hashtable();
        private int count = 0;
        public string startRef = " ";

        public void Crawl(TextBox t)
        {
            //Console.WriteLine("开始爬行");
            t.Text = t.Text + "开始爬行";
            urls.Add(startRef, false);

            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys)
                {
                    if ((bool)urls[url]) continue;
                    current = url;
                }
                if ((current == null) || count > 10) break;
                //Console.WriteLine("爬行" + current + "页面");
                t.Text = t.Text + Environment.NewLine + Environment.NewLine + Environment.NewLine + "爬行" + current + "页面";

                string html = Download(current, t);

                t.Text = t.Text + Environment.NewLine + html;

                urls[current] = true;
                count++;
                if(count==1)Parse(html);//仅爬取第一个网页
            }
            //Console.WriteLine("爬行结束");
            t.Text = t.Text + Environment.NewLine +  "爬行结束";

        }

        public string Download(string url,TextBox t)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);

                string fileName = count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                //Console.WriteLine(ex.Message);
                //显示错误的URL信息
                t.Text = t.Text + Environment.NewLine + ex.Message;
                return "";
            }
        }

        public void Parse(string html)
        {
            //爬取所有网址
            string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(html);

            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', ' ', '>');
                if (strRef.Length == 0) continue;
                if (!strRef.Contains(".html")) continue;//不是html页不加入爬取队列中
                
                //相对地址转换为绝对地址
                if (strRef[0] == '/')
                {
                    strRef = new Uri(new Uri(startRef), strRef).ToString();
                }


                if (urls[strRef] == null) urls[strRef] = false;
            }
        }
        /*static void Main(string[] args)
        {
            Console.WriteLine("Hello World!");
        }*/
    }
}
